Kernel Author:
Bhishan Poudel, Data Scientist, Ph.D Astrophysics .
Bhishan Poudel, Data Scientist, Ph.D Astrophysics .
Data Description¶
Reference: https://www.kaggle.com/c/web-traffic-time-series-forecasting/data
I have cleaned the kaggle wikipedia traffic data and selected only data of 2016 with fraction of 0.1.
The data was melted and additional columns were created.
In [1]:
import numpy as np
import pandas as pd
import seaborn as sns
sns.set(color_codes=True)
import matplotlib
import matplotlib.pyplot as plt
%matplotlib inline
%config InlineBackend.figure_format = 'retina'
sns.set(context='notebook', style='whitegrid', rc={'figure.figsize': (12,8)})
plt.style.use('fivethirtyeight') # better than sns styles.
matplotlib.rcParams['figure.figsize'] = 12,8
import os
import time
# random state
random_state=100
np.random.seed(random_state)
# Jupyter notebook settings for pandas
#pd.set_option('display.float_format', '{:,.2g}'.format) # numbers sep by comma
from pandas.api.types import CategoricalDtype
np.set_printoptions(precision=3)
pd.set_option('display.max_columns', 100)
pd.set_option('display.max_rows', 100) # None for all the rows
pd.set_option('display.max_colwidth', 200)
import IPython
from IPython.display import display, HTML, Image, Markdown
print([(x.__name__,x.__version__) for x in [np, pd,sns,matplotlib]])
In [2]:
import dask
import dask.dataframe as dd
import gc
In [3]:
%%javascript
IPython.OutputArea.auto_scroll_threshold = 9999;
Useful Scripts¶
In [4]:
def show_method_attributes(method, ncols=7,start=None):
""" Show all the attributes of a given method.
Example:
========
show_method_attributes(list)
"""
x = [I for I in dir(method) if I[0]!='_' ]
x = [I for I in x
if I not in 'os np pd sys time psycopg2'.split() ]
if start:
x = [I for I in x if I.startswith(start)]
return pd.DataFrame(np.array_split(x,ncols)).T.fillna('')
Load the data¶
In [5]:
df = pd.read_csv('../../data/wiki/processed/data_cleaned_2016_frac01.csv',
parse_dates=['date'])
print(df.shape) # 5.3 million rows, 21 cols
df.head()
Out[5]:
Memory Reduction¶
In [6]:
df.dtypes
Out[6]:
In [7]:
df.memory_usage(deep=True).sum() * 1e-6 # MB
Out[7]:
In [8]:
# all the year is 2016,drop it.
df.drop('year',axis=1,inplace=True)
In [9]:
cols_int = ['visits']
cols_cat = ['month','day','quarter','day_name','month_name',
'project','access','agent','language']
cols_float = ['mean','median']
for c in cols_int:
df[c] = df[c].astype(np.int32)
for c in cols_float:
df[c] = df[c].astype(np.float32)
for c in cols_cat:
df[c] = df[c].astype(pd.api.types.CategoricalDtype())
In [10]:
# make day_name and month_name ordered categorical
df['day_name'].unique()
Out[10]:
In [11]:
cats = ['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday', 'Sunday']
df['day_name'] = pd.Categorical(df['day_name'], ordered=True, categories=cats)
df['day_name'].unique()
Out[11]:
In [12]:
df['month_name'].unique()
Out[12]:
In [13]:
show_method_attributes(df['month_name'].unique())
Out[13]:
In [14]:
df['month_name'].unique().categories
Out[14]:
In [15]:
cats = ['January', 'February', 'March', 'April', 'May', 'June', 'July',
'August', 'September', 'October', 'November', 'December']
df['month_name'] = pd.Categorical(df['month_name'], ordered=True, categories=cats)
df['month_name'].unique()
Out[15]:
In [16]:
df.memory_usage(deep=True).sum() * 1e-6 # MB
Out[16]:
In [17]:
for c in cols_cat:
print(c)
print(df[c].value_counts().sort_index())
print()
Exploratory Data Analysis (EDA)¶
Most visited page¶
In [18]:
print(df.shape)
df.head()
Out[18]:
In [19]:
df['Page'].nunique() # there are 14.5k unique pages visited in 2016
Out[19]:
In [20]:
df.groupby('Page')['visits'].sum()
Out[20]:
In [154]:
df.groupby('Page')['visits'].sum().sort_values(ascending=False)
Out[154]:
In [153]:
df.groupby('Page')['visits'].sum().nlargest(5)
Out[153]:
In [156]:
df.query(""" Page == 'Special:Search_en.wikipedia.org_desktop_all-agents' """).head()
Out[156]:
Top 5 pages per language¶
In [13]:
# df.groupby('language')['visits'].nlargest(5)
In [14]:
# df.groupby('language')['visits'].apply(lambda x: x.nlargest(5))
In [15]:
# df.groupby('language')['visits'].apply(lambda x: x.nlargest(5).index)
In [16]:
df.groupby('language')['visits'].apply(lambda x: df.loc[x.nlargest(5).index])
Out[16]:
Data Visualizations¶
Single Timeseries Visualization¶
In [157]:
idx = df.groupby('Page')['visits'].sum().idxmax()
df.query(""" Page == @idx """).head()
Out[157]:
In [158]:
ts = df.query(""" Page == @idx """)[['date','visits']].set_index('date')
print(ts.shape)
ts.head()
Out[158]:
In [159]:
ts.plot()
# ts is periodic
# ts has some very large peaks
# ts in not going upward, it does not have trend (it may have if I have more years)
Out[159]:
In [163]:
ts.groupby(ts.index.month).plot();
Language montly mean¶
In [17]:
df.head(2)
Out[17]:
In [21]:
fname_lang_monthly_mean = '../reports/figures/2016_sample001_monthly_visits.png'
if not os.path.isfile(fname_lang_monthly_mean):
plt.figure(figsize=(12,12))
sns.pointplot(x="month_name", y="visits", hue='language', data=df,estimator='mean')
plt.savefig(fname_lang_monthly_mean, dpi=300)
In [19]:
Image(fname_lang_monthly_mean)
Out[19]:
In [20]:
# df.groupby(['month_name', 'language'])['visits'].sum().unstack().reset_index()
# TypeError: cannot insert an item into a CategoricalIndex that is not already an existing category
In [21]:
df.groupby(['month_name', 'language'])['visits'].mean().unstack()
Out[21]:
In [22]:
ax = df.groupby(['month_name', 'language'])['visits'].mean().unstack()\
.reset_index(drop=True).plot(figsize=(12,12), logy=False )
plt.xticks(range(12), rotation=90)
ax.set_xticklabels(df['month_name'].unique());
In [23]:
# exclude english and compare others
In [24]:
# plt.figure(figsize=(12,12))
# df.query("""language != 'English'""").pipe((sns.pointplot,'data'), x='month_name',
# y='visits',hue='language')
# # ValueError: 'c' argument has 12 elements, which is not acceptable for use with 'x' with size 0, 'y' with size 0.
In [25]:
ax = df.query("""language != 'English'""")\
.groupby(['month_name', 'language'])['visits'].mean().unstack()\
.reset_index(drop=True).plot(figsize=(12,12), logy=False )
plt.xticks(range(12), rotation=90)
ax.set_xticklabels(df['month_name'].unique());
Timeseries per language¶
In [22]:
df.groupby(['dayofyear', 'language'])['visits'].mean().unstack().plot(figsize=(12,12))
plt.savefig('../reports/figures/daily_visits.png',dpi=300)
In [28]:
%%time
# plt.figure(figsize=(12,12))
# sns.lineplot(x='dayofyear',y='visits',hue='language',data=df)
# Wall time: 2min 27s
Page Visits per Week Day¶
In [128]:
df['day_name'].unique().categories
Out[128]:
In [23]:
%%time
sns.barplot(x='day_name',y='visits',data=df,
order=['Monday', 'Tuesday', 'Wednesday', 'Thursday', 'Friday', 'Saturday',
'Sunday'])
plt.savefig('../reports/figures/visits_by_weekday.png')
In [77]:
%%time
df.groupby('day_name')['visits'].mean().sort_index().plot.bar()
Out[77]:
In [89]:
%%time
ax = df.groupby('day_name')['visits'].mean().sort_index(ascending=False)\
.plot.barh(use_index=True, color=sns.color_palette('husl',7))
plt.tick_params(axis='y', which='both', labelright='on')
In [99]:
df.groupby(['day_name','month_name'])['visits'].mean().sort_index(ascending=False).unstack().plot.bar()
Out[99]:
In [24]:
%%time
sns.catplot(data=df, kind='bar',ci=None,
x='day_name',y='visits',hue='month_name')
plt.xticks(rotation=90)
plt.savefig('../reports/figures/visits_per_month_per_weekday.png')
In [101]:
df.groupby(['day_name','month_name'])['visits'].mean().sort_index(ascending=False).unstack(0).plot.bar()
Out[101]:
Page Visits per Month Day¶
In [25]:
df1 = df.groupby(["day_name", "day"])['visits'].mean().reset_index()\
.pivot('day','day_name','visits').dropna()
fig, ax = plt.subplots(figsize=(50, 30))
sns.heatmap(data=df1, annot=False, ax=ax, fmt="d", linewidths=2).invert_yaxis()
plt.title('Web Traffic per Days of Week',fontsize=28)
plt.xlabel('Week Day Name', fontsize=28)
plt.ylabel('Day of Month', fontsize=28)
plt.xticks(fontsize=28)
plt.yticks(fontsize=28)
cbar = ax.collections[0].colorbar
cbar.ax.tick_params(labelsize=28)
plt.savefig('../reports/figures/visits_per_dayofmonth_per_dayofweek.png')
plt.show()
In [125]:
df1 = df.groupby(["month_name", "day"])['visits'].mean().reset_index()\
.pivot('day','month_name','visits').dropna()
fig, ax = plt.subplots(figsize=(50, 30))
# sns.set(font_scale=3)
sns.heatmap(data=df1, annot=False, ax=ax, fmt="d", linewidths=2).invert_yaxis()
plt.title('Web Traffic for Months per days of month',fontsize=28)
plt.xlabel('Month Name', fontsize=28)
plt.ylabel('Day of Month', fontsize=28)
plt.xticks(fontsize=28)
plt.yticks(fontsize=28)
cbar = ax.collections[0].colorbar
cbar.ax.tick_params(labelsize=28)
plt.show()
Fast Fourier Transform (FFT)¶
Resources:
- https://ipython-books.github.io/101-analyzing-the-frequency-components-of-a-signal-with-a-fast-fourier-transform/
- https://www.kaggle.com/muonneutrino/wikipedia-traffic-data-exploration
Here we can that the plots seems periodic in nature in time domain. We can work in the frequency domain using FFT transformation of the time series. Peaks in the FFT show us the strongest frequencies in the periodic signal.
The Fourier transform is an alternative representation of a signal as a superposition of periodic components. It is an important mathematical result that any well-behaved function can be represented under this form. Whereas a time-varying signal is most naturally considered as a function of time, the Fourier transform represents it as a function of the frequency. A magnitude and a phase, which are both encoded in a single complex number, are associated to each frequency.
The Discrete Fourier Transform
Let's consider a digital signal x represented by a vector $(x0,...,xN−1)$. We assume that this signal is regularly sampled. The Discrete Fourier Transform (DFT) of x is $X=(X0,...,XN−1)$ defined as:
The DFT can be computed efficiently with the Fast Fourier Transform (FFT), an algorithm that exploits symmetries and redundancies in this definition to considerably speed up the computation. The complexity of the FFT is $O(NlogN)$ instead of $O(N^2)$ for the naive DFT. The FFT is one of the most important algorithms of the digital universe.
In [43]:
days = df['dayofyear'].unique()
In [40]:
df_daily = df.groupby(['dayofyear', 'language'])['visits'].mean().unstack()
df_daily.head()
Out[40]:
In [41]:
df_daily.columns
Out[41]:
In [44]:
def plot_with_fft(df_daily, col):
from scipy.fftpack import fft
fig = plt.figure(1,figsize=[15,5])
plt.ylabel('Views per Page')
plt.xlabel('Day')
plt.title(col)
plt.xticks(range(0,370,10))
plt.plot(days, df_daily[col].to_numpy(),label = col )
fig = plt.figure(2,figsize=[15,5])
fft_complex = fft(df_daily[col].to_numpy())
fft_mag = [np.sqrt(np.real(x)*np.real(x)+np.imag(x)*np.imag(x))
for x in fft_complex]
fft_xvals = [day / days[-1] for day in days]
npts = len(fft_xvals) // 2 + 1
fft_mag = fft_mag[:npts]
fft_xvals = fft_xvals[:npts]
plt.ylabel('FFT Magnitude')
plt.xlabel(r"Frequency [days]$^{-1}$")
plt.title('Fourier Transform')
plt.plot(fft_xvals[1:],fft_mag[1:],label = col )
# Draw lines at 1, 1/2, and 1/3 week periods
plt.axvline(x=1./7,color='red',alpha=0.3)
plt.axvline(x=2./7,color='red',alpha=0.3)
plt.axvline(x=3./7,color='red',alpha=0.3)
plt.show()
for col in df_daily.columns:
plot_with_fft(df_daily, col)
In [ ]: